{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../../\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import tabulate\n",
    "import pprint\n",
    "import click\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from ray.tune.commands import *\n",
    "from nupic.research.frameworks.dynamic_sparse.common.browser import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True /Users/mcaporale/nta/results/run_stochastic_synapses_gscvalidate\n"
     ]
    }
   ],
   "source": [
    "base = 'run_stochastic_synapses_gscvalidate'\n",
    "exps = [\n",
    "    base\n",
    "]\n",
    "    \n",
    "paths = [os.path.expanduser(\"~/nta/results/{}\".format(e)) for e in exps]\n",
    "for p in paths:\n",
    "    print(os.path.exists(p), p)\n",
    "has_nz = lambda k: \"nz\" in k\n",
    "has_sp = lambda k: \"sparsity\" in k\n",
    "df = load_many(paths, raw_metrics=[has_nz, has_sp])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Experiment Name</th>\n",
       "      <th>train_acc_max</th>\n",
       "      <th>train_acc_max_epoch</th>\n",
       "      <th>train_acc_min</th>\n",
       "      <th>train_acc_min_epoch</th>\n",
       "      <th>train_acc_median</th>\n",
       "      <th>train_acc_last</th>\n",
       "      <th>val_acc_max</th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th>val_acc_min</th>\n",
       "      <th>...</th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>lr_gamma</th>\n",
       "      <th>lr_scheduler</th>\n",
       "      <th>lr_step_size</th>\n",
       "      <th>model</th>\n",
       "      <th>name</th>\n",
       "      <th>network</th>\n",
       "      <th>optim_alg</th>\n",
       "      <th>test_noise</th>\n",
       "      <th>use_tqdm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_model=StochasticSynapsesModel,network=gsc_bi...</td>\n",
       "      <td>0.970364</td>\n",
       "      <td>98</td>\n",
       "      <td>0.335465</td>\n",
       "      <td>0</td>\n",
       "      <td>0.950786</td>\n",
       "      <td>0.969534</td>\n",
       "      <td>0.969126</td>\n",
       "      <td>89</td>\n",
       "      <td>0.631516</td>\n",
       "      <td>...</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.9825</td>\n",
       "      <td>StepLR</td>\n",
       "      <td>1</td>\n",
       "      <td>StochasticSynapsesModel</td>\n",
       "      <td>run_stochastic_synapses_gscvalidate</td>\n",
       "      <td>gsc_binary_cnn</td>\n",
       "      <td>Adam</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1_model=StochasticSynapsesModel,network=gsc_ha...</td>\n",
       "      <td>0.960600</td>\n",
       "      <td>93</td>\n",
       "      <td>0.441021</td>\n",
       "      <td>0</td>\n",
       "      <td>0.944610</td>\n",
       "      <td>0.960453</td>\n",
       "      <td>0.969928</td>\n",
       "      <td>85</td>\n",
       "      <td>0.720930</td>\n",
       "      <td>...</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.9825</td>\n",
       "      <td>StepLR</td>\n",
       "      <td>1</td>\n",
       "      <td>StochasticSynapsesModel</td>\n",
       "      <td>run_stochastic_synapses_gscvalidate</td>\n",
       "      <td>gsc_hard_concrete_cnn</td>\n",
       "      <td>Adam</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2_model=StochasticSynapsesModel,network=gsc_bi...</td>\n",
       "      <td>0.968607</td>\n",
       "      <td>97</td>\n",
       "      <td>0.334635</td>\n",
       "      <td>0</td>\n",
       "      <td>0.949907</td>\n",
       "      <td>0.965677</td>\n",
       "      <td>0.965116</td>\n",
       "      <td>82</td>\n",
       "      <td>0.497594</td>\n",
       "      <td>...</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.9825</td>\n",
       "      <td>StepLR</td>\n",
       "      <td>1</td>\n",
       "      <td>StochasticSynapsesModel</td>\n",
       "      <td>run_stochastic_synapses_gscvalidate</td>\n",
       "      <td>gsc_binary_cnn</td>\n",
       "      <td>Adam</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3_model=StochasticSynapsesModel,network=gsc_ha...</td>\n",
       "      <td>0.960111</td>\n",
       "      <td>97</td>\n",
       "      <td>0.395713</td>\n",
       "      <td>0</td>\n",
       "      <td>0.941192</td>\n",
       "      <td>0.959037</td>\n",
       "      <td>0.969126</td>\n",
       "      <td>78</td>\n",
       "      <td>0.551724</td>\n",
       "      <td>...</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.9825</td>\n",
       "      <td>StepLR</td>\n",
       "      <td>1</td>\n",
       "      <td>StochasticSynapsesModel</td>\n",
       "      <td>run_stochastic_synapses_gscvalidate</td>\n",
       "      <td>gsc_hard_concrete_cnn</td>\n",
       "      <td>Adam</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4_model=StochasticSynapsesModel,network=gsc_bi...</td>\n",
       "      <td>0.969241</td>\n",
       "      <td>97</td>\n",
       "      <td>0.324773</td>\n",
       "      <td>0</td>\n",
       "      <td>0.951323</td>\n",
       "      <td>0.964310</td>\n",
       "      <td>0.966319</td>\n",
       "      <td>78</td>\n",
       "      <td>0.611468</td>\n",
       "      <td>...</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.9825</td>\n",
       "      <td>StepLR</td>\n",
       "      <td>1</td>\n",
       "      <td>StochasticSynapsesModel</td>\n",
       "      <td>run_stochastic_synapses_gscvalidate</td>\n",
       "      <td>gsc_binary_cnn</td>\n",
       "      <td>Adam</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5_model=StochasticSynapsesModel,network=gsc_ha...</td>\n",
       "      <td>0.960795</td>\n",
       "      <td>93</td>\n",
       "      <td>0.430183</td>\n",
       "      <td>0</td>\n",
       "      <td>0.944781</td>\n",
       "      <td>0.958940</td>\n",
       "      <td>0.971532</td>\n",
       "      <td>96</td>\n",
       "      <td>0.622294</td>\n",
       "      <td>...</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.9825</td>\n",
       "      <td>StepLR</td>\n",
       "      <td>1</td>\n",
       "      <td>StochasticSynapsesModel</td>\n",
       "      <td>run_stochastic_synapses_gscvalidate</td>\n",
       "      <td>gsc_hard_concrete_cnn</td>\n",
       "      <td>Adam</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6_model=StochasticSynapsesModel,network=gsc_bi...</td>\n",
       "      <td>0.966849</td>\n",
       "      <td>90</td>\n",
       "      <td>0.348111</td>\n",
       "      <td>0</td>\n",
       "      <td>0.950713</td>\n",
       "      <td>0.965775</td>\n",
       "      <td>0.967121</td>\n",
       "      <td>84</td>\n",
       "      <td>0.619487</td>\n",
       "      <td>...</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.9825</td>\n",
       "      <td>StepLR</td>\n",
       "      <td>1</td>\n",
       "      <td>StochasticSynapsesModel</td>\n",
       "      <td>run_stochastic_synapses_gscvalidate</td>\n",
       "      <td>gsc_binary_cnn</td>\n",
       "      <td>Adam</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7_model=StochasticSynapsesModel,network=gsc_ha...</td>\n",
       "      <td>0.959916</td>\n",
       "      <td>98</td>\n",
       "      <td>0.428669</td>\n",
       "      <td>0</td>\n",
       "      <td>0.943243</td>\n",
       "      <td>0.958402</td>\n",
       "      <td>0.969126</td>\n",
       "      <td>96</td>\n",
       "      <td>0.626704</td>\n",
       "      <td>...</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.9825</td>\n",
       "      <td>StepLR</td>\n",
       "      <td>1</td>\n",
       "      <td>StochasticSynapsesModel</td>\n",
       "      <td>run_stochastic_synapses_gscvalidate</td>\n",
       "      <td>gsc_hard_concrete_cnn</td>\n",
       "      <td>Adam</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8_model=StochasticSynapsesModel,network=gsc_bi...</td>\n",
       "      <td>0.968362</td>\n",
       "      <td>96</td>\n",
       "      <td>0.347036</td>\n",
       "      <td>0</td>\n",
       "      <td>0.948858</td>\n",
       "      <td>0.964945</td>\n",
       "      <td>0.965918</td>\n",
       "      <td>82</td>\n",
       "      <td>0.538492</td>\n",
       "      <td>...</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.9825</td>\n",
       "      <td>StepLR</td>\n",
       "      <td>1</td>\n",
       "      <td>StochasticSynapsesModel</td>\n",
       "      <td>run_stochastic_synapses_gscvalidate</td>\n",
       "      <td>gsc_binary_cnn</td>\n",
       "      <td>Adam</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9_model=StochasticSynapsesModel,network=gsc_ha...</td>\n",
       "      <td>0.960111</td>\n",
       "      <td>97</td>\n",
       "      <td>0.416268</td>\n",
       "      <td>0</td>\n",
       "      <td>0.943267</td>\n",
       "      <td>0.958451</td>\n",
       "      <td>0.968725</td>\n",
       "      <td>71</td>\n",
       "      <td>0.705293</td>\n",
       "      <td>...</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.9825</td>\n",
       "      <td>StepLR</td>\n",
       "      <td>1</td>\n",
       "      <td>StochasticSynapsesModel</td>\n",
       "      <td>run_stochastic_synapses_gscvalidate</td>\n",
       "      <td>gsc_hard_concrete_cnn</td>\n",
       "      <td>Adam</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 54 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Experiment Name  train_acc_max  \\\n",
       "0  0_model=StochasticSynapsesModel,network=gsc_bi...       0.970364   \n",
       "1  1_model=StochasticSynapsesModel,network=gsc_ha...       0.960600   \n",
       "2  2_model=StochasticSynapsesModel,network=gsc_bi...       0.968607   \n",
       "3  3_model=StochasticSynapsesModel,network=gsc_ha...       0.960111   \n",
       "4  4_model=StochasticSynapsesModel,network=gsc_bi...       0.969241   \n",
       "5  5_model=StochasticSynapsesModel,network=gsc_ha...       0.960795   \n",
       "6  6_model=StochasticSynapsesModel,network=gsc_bi...       0.966849   \n",
       "7  7_model=StochasticSynapsesModel,network=gsc_ha...       0.959916   \n",
       "8  8_model=StochasticSynapsesModel,network=gsc_bi...       0.968362   \n",
       "9  9_model=StochasticSynapsesModel,network=gsc_ha...       0.960111   \n",
       "\n",
       "   train_acc_max_epoch  train_acc_min  train_acc_min_epoch  train_acc_median  \\\n",
       "0                   98       0.335465                    0          0.950786   \n",
       "1                   93       0.441021                    0          0.944610   \n",
       "2                   97       0.334635                    0          0.949907   \n",
       "3                   97       0.395713                    0          0.941192   \n",
       "4                   97       0.324773                    0          0.951323   \n",
       "5                   93       0.430183                    0          0.944781   \n",
       "6                   90       0.348111                    0          0.950713   \n",
       "7                   98       0.428669                    0          0.943243   \n",
       "8                   96       0.347036                    0          0.948858   \n",
       "9                   97       0.416268                    0          0.943267   \n",
       "\n",
       "   train_acc_last  val_acc_max  val_acc_max_epoch  val_acc_min  ...  \\\n",
       "0        0.969534     0.969126                 89     0.631516  ...   \n",
       "1        0.960453     0.969928                 85     0.720930  ...   \n",
       "2        0.965677     0.965116                 82     0.497594  ...   \n",
       "3        0.959037     0.969126                 78     0.551724  ...   \n",
       "4        0.964310     0.966319                 78     0.611468  ...   \n",
       "5        0.958940     0.971532                 96     0.622294  ...   \n",
       "6        0.965775     0.967121                 84     0.619487  ...   \n",
       "7        0.958402     0.969126                 96     0.626704  ...   \n",
       "8        0.964945     0.965918                 82     0.538492  ...   \n",
       "9        0.958451     0.968725                 71     0.705293  ...   \n",
       "\n",
       "   learning_rate  lr_gamma  lr_scheduler lr_step_size  \\\n",
       "0           0.01    0.9825        StepLR            1   \n",
       "1           0.01    0.9825        StepLR            1   \n",
       "2           0.01    0.9825        StepLR            1   \n",
       "3           0.01    0.9825        StepLR            1   \n",
       "4           0.01    0.9825        StepLR            1   \n",
       "5           0.01    0.9825        StepLR            1   \n",
       "6           0.01    0.9825        StepLR            1   \n",
       "7           0.01    0.9825        StepLR            1   \n",
       "8           0.01    0.9825        StepLR            1   \n",
       "9           0.01    0.9825        StepLR            1   \n",
       "\n",
       "                     model                                 name  \\\n",
       "0  StochasticSynapsesModel  run_stochastic_synapses_gscvalidate   \n",
       "1  StochasticSynapsesModel  run_stochastic_synapses_gscvalidate   \n",
       "2  StochasticSynapsesModel  run_stochastic_synapses_gscvalidate   \n",
       "3  StochasticSynapsesModel  run_stochastic_synapses_gscvalidate   \n",
       "4  StochasticSynapsesModel  run_stochastic_synapses_gscvalidate   \n",
       "5  StochasticSynapsesModel  run_stochastic_synapses_gscvalidate   \n",
       "6  StochasticSynapsesModel  run_stochastic_synapses_gscvalidate   \n",
       "7  StochasticSynapsesModel  run_stochastic_synapses_gscvalidate   \n",
       "8  StochasticSynapsesModel  run_stochastic_synapses_gscvalidate   \n",
       "9  StochasticSynapsesModel  run_stochastic_synapses_gscvalidate   \n",
       "\n",
       "                 network optim_alg test_noise use_tqdm  \n",
       "0         gsc_binary_cnn      Adam      False     True  \n",
       "1  gsc_hard_concrete_cnn      Adam      False     True  \n",
       "2         gsc_binary_cnn      Adam      False     True  \n",
       "3  gsc_hard_concrete_cnn      Adam      False     True  \n",
       "4         gsc_binary_cnn      Adam      False     True  \n",
       "5  gsc_hard_concrete_cnn      Adam      False     True  \n",
       "6         gsc_binary_cnn      Adam      False     True  \n",
       "7  gsc_hard_concrete_cnn      Adam      False     True  \n",
       "8         gsc_binary_cnn      Adam      False     True  \n",
       "9  gsc_hard_concrete_cnn      Adam      False     True  \n",
       "\n",
       "[10 rows x 54 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Experiment Name', 'train_acc_max', 'train_acc_max_epoch',\n",
       "       'train_acc_min', 'train_acc_min_epoch', 'train_acc_median',\n",
       "       'train_acc_last', 'val_acc_max', 'val_acc_max_epoch', 'val_acc_min',\n",
       "       'val_acc_min_epoch', 'val_acc_median', 'val_acc_last', 'val_acc_all',\n",
       "       'cnn1/hist_expected_nz_by_unit', 'cnn1/expected_nz',\n",
       "       'cnn1/hist_inference_nz_by_unit', 'cnn1/inference_nz',\n",
       "       'cnn2/hist_expected_nz_by_unit', 'cnn2/expected_nz',\n",
       "       'cnn2/hist_inference_nz_by_unit', 'cnn2/inference_nz',\n",
       "       'fc1/hist_expected_nz_by_unit', 'fc1/expected_nz',\n",
       "       'fc1/hist_inference_nz_by_unit', 'fc1/inference_nz',\n",
       "       'fc2/hist_expected_nz_by_unit', 'fc2/expected_nz',\n",
       "       'fc2/hist_inference_nz_by_unit', 'fc2/inference_nz', 'epochs',\n",
       "       'experiment_file_name', 'trial_time', 'mean_epoch_time',\n",
       "       'batch_size_test', 'batch_size_train', 'checkpoint_dir', 'data_dir',\n",
       "       'dataset_name', 'debug_sparse', 'debug_weights', 'device',\n",
       "       'droprate_init', 'l0_strength', 'learning_rate', 'lr_gamma',\n",
       "       'lr_scheduler', 'lr_step_size', 'model', 'name', 'network', 'optim_alg',\n",
       "       'test_noise', 'use_tqdm'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper functions\n",
    "def mean_and_std(s):\n",
    "    return \"{:.3f} ± {:.3f}\".format(s.mean(), s.std())\n",
    "\n",
    "def round_mean(s):\n",
    "    return \"{:.0f}\".format(round(s.mean()))\n",
    "\n",
    "stats = ['min', 'max', 'mean', 'std']\n",
    "\n",
    "def agg(columns, filter=None, round=3):\n",
    "    if filter is None:\n",
    "        return (df.groupby(columns)\n",
    "             .agg({'val_acc_max_epoch': round_mean,\n",
    "                   'val_acc_max': stats,                \n",
    "                   'model': ['count']})).round(round)\n",
    "    else:\n",
    "        return (df[filter].groupby(columns)\n",
    "             .agg({'val_acc_max_epoch': round_mean,\n",
    "                   'val_acc_max': stats,                \n",
    "                   'model': ['count']})).round(round)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    None\n",
       "1    None\n",
       "2    None\n",
       "3    None\n",
       "4    None\n",
       "5    None\n",
       "6    None\n",
       "7    None\n",
       "8    None\n",
       "9    None\n",
       "Name: cnn1/expected_nz, dtype: object"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['cnn1/expected_nz'].apply(lambda s: print(len(s)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['cnn1/final_expected_nz'] = df['cnn1/expected_nz'].apply(lambda s: s.iloc[0])\n",
    "df['cnn1/final_inference_nz'] = df['cnn1/inference_nz'].apply(lambda s: s.iloc[0])\n",
    "df['cnn2/final_expected_nz'] = df['cnn2/expected_nz'].apply(lambda s: s.iloc[0])\n",
    "df['cnn2/final_inference_nz'] = df['cnn2/inference_nz'].apply(lambda s: s.iloc[0])\n",
    "df['fc1/final_expected_nz'] = df['fc1/expected_nz'].apply(lambda s: s.iloc[0])\n",
    "df['fc1/final_inference_nz'] = df['fc1/inference_nz'].apply(lambda s: s.iloc[0])\n",
    "df['fc2/final_expected_nz'] = df['fc2/expected_nz'].apply(lambda s: s.iloc[0])\n",
    "df['fc2/final_inference_nz'] = df['fc2/inference_nz'].apply(lambda s: s.iloc[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">val_acc_max</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>network</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>gsc_binary_cnn</th>\n",
       "      <td>0.966720</td>\n",
       "      <td>0.001527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gsc_hard_concrete_cnn</th>\n",
       "      <td>0.969687</td>\n",
       "      <td>0.001120</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      val_acc_max          \n",
       "                             mean       std\n",
       "network                                    \n",
       "gsc_binary_cnn           0.966720  0.001527\n",
       "gsc_hard_concrete_cnn    0.969687  0.001120"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby([\"network\"]).agg({\n",
    "    \"val_acc_max\": [\"mean\", \"std\"],\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cnn1/final_expected_nz</th>\n",
       "      <th>cnn2/final_expected_nz</th>\n",
       "      <th>fc1/final_expected_nz</th>\n",
       "      <th>fc2/final_expected_nz</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>network</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>gsc_binary_cnn</th>\n",
       "      <td>1565.860693</td>\n",
       "      <td>99951.171875</td>\n",
       "      <td>1567201.325</td>\n",
       "      <td>10607.786133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gsc_hard_concrete_cnn</th>\n",
       "      <td>1497.316968</td>\n",
       "      <td>95709.403125</td>\n",
       "      <td>1504765.425</td>\n",
       "      <td>10139.535156</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       cnn1/final_expected_nz  cnn2/final_expected_nz  \\\n",
       "network                                                                 \n",
       "gsc_binary_cnn                    1565.860693            99951.171875   \n",
       "gsc_hard_concrete_cnn             1497.316968            95709.403125   \n",
       "\n",
       "                       fc1/final_expected_nz  fc2/final_expected_nz  \n",
       "network                                                              \n",
       "gsc_binary_cnn                   1567201.325           10607.786133  \n",
       "gsc_hard_concrete_cnn            1504765.425           10139.535156  "
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby([\"network\"]).agg({\n",
    "    'cnn1/final_expected_nz': \"mean\",\n",
    "    'cnn2/final_expected_nz': \"mean\",\n",
    "    'fc1/final_expected_nz': \"mean\",\n",
    "    'fc2/final_expected_nz': \"mean\",\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cnn1/final_inference_nz</th>\n",
       "      <th>cnn2/final_inference_nz</th>\n",
       "      <th>fc1/final_inference_nz</th>\n",
       "      <th>fc2/final_inference_nz</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>network</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>gsc_binary_cnn</th>\n",
       "      <td>1599.8</td>\n",
       "      <td>102399.8</td>\n",
       "      <td>1599983.6</td>\n",
       "      <td>11731.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gsc_hard_concrete_cnn</th>\n",
       "      <td>1600.0</td>\n",
       "      <td>102399.8</td>\n",
       "      <td>1600000.0</td>\n",
       "      <td>11231.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       cnn1/final_inference_nz  cnn2/final_inference_nz  \\\n",
       "network                                                                   \n",
       "gsc_binary_cnn                          1599.8                 102399.8   \n",
       "gsc_hard_concrete_cnn                   1600.0                 102399.8   \n",
       "\n",
       "                       fc1/final_inference_nz  fc2/final_inference_nz  \n",
       "network                                                                \n",
       "gsc_binary_cnn                      1599983.6                 11731.6  \n",
       "gsc_hard_concrete_cnn               1600000.0                 11231.2  "
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby([\"network\"]).agg({\n",
    "    'cnn1/final_inference_nz': \"mean\",\n",
    "    'cnn2/final_inference_nz': \"mean\",\n",
    "    'fc1/final_inference_nz': \"mean\",\n",
    "    'fc2/final_inference_nz': \"mean\",\n",
    "})"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
